{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mega Parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Started parsing the file under job_id e5e0367d-2f83-4e4d-84e5-4d5df7119516\n",
      "Started parsing the file under job_id 0b5d66aa-bbab-454b-b256-82495d20f91f\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at microsoft/table-transformer-structure-recognition were not used when initializing TableTransformerForObjectDetection: ['model.backbone.conv_encoder.model.layer2.0.downsample.1.num_batches_tracked', 'model.backbone.conv_encoder.model.layer3.0.downsample.1.num_batches_tracked', 'model.backbone.conv_encoder.model.layer4.0.downsample.1.num_batches_tracked']\n",
      "- This IS expected if you are initializing TableTransformerForObjectDetection from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TableTransformerForObjectDetection from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from megaparse.Converter import MegaParse\n",
    "import os \n",
    "\n",
    "api_key: str | None = os.getenv(\"LLAMA_CLOUD_API_KEY\")\n",
    "\n",
    "converter = MegaParse(file_path=\"../megaparse/tests/input_tests/MegaFake_report.pdf\", llama_parse_api_key=api_key)\n",
    "md_content = converter.convert()\n",
    "converter.save_md(md_content, Path(\"../megaparse/tests/output_tests/MegaFake_report_llama_parse_megaparse.md\"))\n",
    "\n",
    "converter = MegaParse(file_path=\"../megaparse/tests/input_tests/MegaFake_report.pdf\", llama_parse_api_key=api_key)\n",
    "md_content = converter.convert(gpt4o_cleaner = True)\n",
    "converter.save_md(md_content, Path(\"../megaparse/tests/output_tests/MegaFake_report_llama_parse_megaparse_gptcleaner.md\"))\n",
    "\n",
    "\n",
    "converter = MegaParse(file_path=\"../megaparse/tests/input_tests/MegaFake_report.pdf\")\n",
    "md_content = converter.convert()\n",
    "converter.save_md(md_content, Path(\"../megaparse/tests/output_tests/MegaFake_report_unstructured_parse_megaparse.md\"))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LLama Parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Started parsing the file under job_id f78ee794-ffde-4e0a-938d-987f1b22cfcb\n"
     ]
    }
   ],
   "source": [
    "from typing import List\n",
    "from llama_index.core.schema import Document\n",
    "import nest_asyncio\n",
    "\n",
    "nest_asyncio.apply()\n",
    "#GET LLAMA_CLOUD_API_KEY\n",
    "import os\n",
    "from llama_parse import LlamaParse\n",
    "from llama_parse.utils import ResultType, Language\n",
    "\n",
    "api_key: str | None = os.getenv(\"LLAMA_CLOUD_API_KEY\")\n",
    "\n",
    "parsing_instructions = \"Do not take into account the page breaks (no --- between pages), do not repeat the header and the footer so the tables are merged. Keep the same format for similar tables.\"\n",
    "\n",
    "parser = LlamaParse(\n",
    "    api_key=str(api_key), \n",
    "    result_type=ResultType.MD,\n",
    "    gpt4o_mode=True,\n",
    "    verbose=True,\n",
    "    language=Language.FRENCH,\n",
    "    parsing_instruction=parsing_instructions,  # Optionally you can define a parsing instruction\n",
    ")\n",
    "# sync\n",
    "documents: List[Document] = parser.load_data(\"../megaparse/tests/input_tests/MegaFake_report.pdf\")\n",
    "\n",
    "with open(\"../megaparse/tests/output_tests/MegaFake_report_llama.md\", \"w\") as f:\n",
    "        f.write(documents[0].get_content())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unstructured"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import UnstructuredPDFLoader\n",
    "loader = UnstructuredPDFLoader(\"../megaparse/tests/input_tests/MegaFake_report.pdf\", strategy=\"hi_res\", infer_table_structure=True,\n",
    ")\n",
    "data = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"../megaparse/tests/output_tests/MegaFake_report_unstructured.md\", \"w\") as f:\n",
    "        f.write(data[0].page_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation with Diff Lib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import difflib\n",
    "def read_file(file_path):\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        return file.readlines()\n",
    "\n",
    "def compare_files(source_path, target_path, with_formatting=False):\n",
    "    source_lines = read_file(source_path)\n",
    "    target_lines = read_file(target_path)\n",
    "    if not with_formatting:\n",
    "        source_lines = [line.replace(\"*\",\"\") for line in source_lines]\n",
    "        target_lines = [line.replace(\"*\",\"\") for line in target_lines]\n",
    "\n",
    "    diff = difflib.unified_diff(\n",
    "    source_lines,\n",
    "    target_lines,\n",
    "    fromfile='target.md',\n",
    "    tofile='generated.md',\n",
    "    lineterm=''\n",
    "    )\n",
    "\n",
    "    modifications = 0\n",
    "    for line in diff:\n",
    "        #print(line)\n",
    "        if line.startswith('+') and not line.startswith('+++'):\n",
    "            modifications += 1\n",
    "        elif line.startswith('-') and not line.startswith('---'):\n",
    "            modifications += 1\n",
    "\n",
    "    return modifications\n",
    "    \n",
    "diff_megaparse_unstructured = compare_files(\"../megaparse/tests/output_tests/MegaFake_report_unstructured_parse_megaparse.md\", \"../megaparse/tests/output_tests/MegaFake_report.md\")\n",
    "diff_megaparse_llama_gptcleaner = compare_files(\"../megaparse/tests/output_tests/MegaFake_report_llama_parse_megaparse_gptcleaner.md\", \"../megaparse/tests/output_tests/MegaFake_report.md\")\n",
    "diff_megaparse_llama = compare_files(\"../megaparse/tests/output_tests/MegaFake_report_llama_parse_megaparse.md\", \"../megaparse/tests/output_tests/MegaFake_report.md\")\n",
    "diff_llamaparse = compare_files(\"../megaparse/tests/output_tests/MegaFake_report_llama.md\", \"../megaparse/tests/output_tests/MegaFake_report.md\")\n",
    "diff_unstructured = compare_files(\"../megaparse/tests/output_tests/MegaFake_report_unstructured.md\", \"../megaparse/tests/output_tests/MegaFake_report.md\")\n",
    "diff_megaparse_llm = compare_files(\"../megaparse/tests/output_tests/MegaFake_report_llm_megaparse.md\", \"../megaparse/tests/output_tests/MegaFake_report.md\")\n",
    "diff_megaparse_unstructured_augmented = compare_files(\"../megaparse/tests/output_tests/MegaFake_report_unstructured_augmented.md\", \"../megaparse/tests/output_tests/MegaFake_report.md\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff_results = {\n",
    "    \"**Megaparse**\": diff_megaparse_unstructured,\n",
    "    \"Megaparse with LLamaParse\": diff_megaparse_llama,\n",
    "    \"Megaparse with LLamaParse and GPTCleaner\": diff_megaparse_llama_gptcleaner,\n",
    "    \"LMM megaparse\": diff_megaparse_llm,\n",
    "    \"LLama Parse\": diff_llamaparse,\n",
    "    \"Unstructured Augmented Parse\": diff_megaparse_unstructured_augmented,\n",
    "}\n",
    "\n",
    "# Sort the results\n",
    "sorted_diff_results = sorted(diff_results.items(), key=lambda x: x[1])\n",
    "\n",
    "# Generate a table with the results\n",
    "benchmark_results = \"| Parser | Diff |\\n|---|---|\\n\"\n",
    "for parser, diff in sorted_diff_results:\n",
    "    benchmark_results += f\"| {parser} | {diff} |\\n\"\n",
    "\n",
    "# Update README.md file\n",
    "with open(\"../README.md\", \"r\") as readme_file:\n",
    "    readme_content = readme_file.read()\n",
    "\n",
    "start_marker = \"<!---BENCHMARK-->\"\n",
    "end_marker = \"<!---END_BENCHMARK-->\"\n",
    "start_index = readme_content.find(start_marker) + len(start_marker)\n",
    "end_index = readme_content.find(end_marker)\n",
    "\n",
    "updated_readme_content = readme_content[:start_index] + \"\\n\" + benchmark_results + readme_content[end_index:]\n",
    "\n",
    "with open(\"../README.md\", \"w\") as readme_file:\n",
    "    readme_file.write(updated_readme_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- target.md\n",
      "+++ generated.md\n",
      "@@ -1,18 +1,19 @@\n",
      "-| My Mega fake | report | #1756394 31/05/2024 |\n",
      "\n",
      "-|--------------|--------|---------------------|\n",
      "\n",
      "-|              |        |                     |\n",
      "\n",
      "+| My Mega fake report | #1756394 | 31/05/2024 |\n",
      "\n",
      "+|---------------------|----------|------------|\n",
      "\n",
      " \n",
      "\n",
      " # Why Mega Parse might be the best ?\n",
      "\n",
      " \n",
      "\n",
      "-# Introduction\n",
      "\n",
      "+## Introduction\n",
      "\n",
      " \n",
      "\n",
      " Mega Parse is a state-of-the-art document parser designed to convert various document formats such as PDF, DOCX, PPTX, and more into Markdown (MD) format, making them ready for Retrieval-Augmented Generation (RAG) ingestion. In today's data-driven world, the ability to efficiently manage and utilize large volumes of information is crucial. This report explores the features, benefits, and comparative performance of Mega Parse, illustrating why it stands out as a superior tool in the realm of document parsing.\n",
      "\n",
      " \n",
      "\n",
      "-# Features of Mega Parse\n",
      "\n",
      "+## Features of Mega Parse\n",
      "\n",
      " \n",
      "\n",
      " Mega Parse boasts an impressive array of features tailored to meet the diverse needs of modern enterprises.\n",
      "\n",
      " \n",
      "\n",
      " Multiple Format Support: Mega Parse supports a wide range of document formats including PDF, DOCX, and PPTX. This versatility allows users to handle various document types without needing multiple tools. Whether you are working with text documents, presentations, or scanned PDFs, Mega Parse has you covered.\n",
      "\n",
      "+\n",
      "\n",
      "+High-Speed Processing: One of the standout features of Mega Parse is its ability to convert documents at a rapid pace. With processing speeds of up to 120 pages per minute, it significantly enhances productivity by reducing the time spent on document conversion.\n",
      "\n",
      " \n",
      "\n",
      " Markdown Output: Mega Parse converts documents into a structured Markdown format. Markdown is a lightweight markup language with plain text formatting syntax, which is widely used because of its simplicity and ease of conversion to other formats. This makes it ideal for RAG ingestion, where structured and easily interpretable data is paramount.\n",
      "\n",
      " \n",
      "\n",
      "@@ -24,7 +25,7 @@\n",
      " \n",
      "\n",
      " Error Handling: Advanced error handling capabilities ensure that any issues encountered during the conversion process are managed effectively, minimizing disruptions and maintaining workflow efficiency.\n",
      "\n",
      " \n",
      "\n",
      "-# Benefits of Mega Parse\n",
      "\n",
      "+## Benefits of Mega Parse\n",
      "\n",
      " \n",
      "\n",
      " The implementation of Mega Parse offers numerous benefits that can transform the way organizations manage their documents.\n",
      "\n",
      " \n",
      "\n",
      "@@ -32,9 +33,7 @@\n",
      " \n",
      "\n",
      " Versatility: Mega Parse's ability to handle multiple document types makes it a versatile tool for various industries. Whether you need to convert legal documents, technical manuals, or business presentations, Mega Parse is equipped to handle the task.\n",
      "\n",
      " \n",
      "\n",
      "-Enhanced Knowledge Management: Converting documents to Markdown facilitates easier content management and retrieval. Markdown files are not only lightweight but\n",
      "\n",
      "-\n",
      "\n",
      "-also highly compatible with various knowledge management systems, making it easier to organize, search, and utilize information.\n",
      "\n",
      "+Enhanced Knowledge Management: Converting documents to Markdown facilitates easier content management and retrieval. Markdown files are not only lightweight but also highly compatible with various knowledge management systems, making it easier to organize, search, and utilize information.\n",
      "\n",
      " \n",
      "\n",
      " Improved Workflow: Mega Parse simplifies the process of preparing documents for machine learning and AI applications. By converting documents into a structured format, it reduces the time and effort required to preprocess data, allowing teams to focus on higher-level tasks.\n",
      "\n",
      " \n",
      "\n",
      "@@ -42,57 +41,45 @@\n",
      " \n",
      "\n",
      " Scalability: Mega Parse is designed to scale with the needs of an organization. As document volumes grow, Mega Parse can handle the increased load without compromising performance, making it a future-proof solution for document management.\n",
      "\n",
      " \n",
      "\n",
      "-# Comparative Performance\n",
      "\n",
      "+## Comparative Performance\n",
      "\n",
      " \n",
      "\n",
      " The following table provides a comprehensive comparative analysis of Mega Parse against other document parsers based on fictional performance metrics. This comparison highlights the strengths of Mega Parse in various key areas.\n",
      "\n",
      " \n",
      "\n",
      "-| Metric                        | Mega Parse           | Parser A   | Parser B   | Parser C   | Parser D          |\n",
      "\n",
      "-|-------------------------------|----------------------|------------|------------|------------|-------------------|\n",
      "\n",
      "-| Supported Formats             | PDF, DOCX, PPTX      | PDF, DOCX  | DOCX, PPTX | PDF, PPTX  | PDF, DOCX, XLSX   |\n",
      "\n",
      "-| Conversion Speed (pages/min)  | 120                  | 90         | 100        | 85         | 95                |\n",
      "\n",
      "-\n",
      "\n",
      "-| Metric                               | Mega Parse | Parser A | Parser B | Parser C | Parser D   | Plain Text  |\n",
      "\n",
      "-|--------------------------------------|------------|----------|----------|----------|------------|-------------|\n",
      "\n",
      "-| Accuracy Rate (%)                    | 98         | 95       | 93       | 90       | 92         | 90          |\n",
      "\n",
      "-| Output Format                        | Markdown   | HTML     | Markdown | HTML     | Plain Text | Plain Text  |\n",
      "\n",
      "-| Error Rate (%)                       | 1          | 3        | 4        | 5        | 3          | 5           |\n",
      "\n",
      "-| Ease of Use                          | High       | Medium   | High     | Medium   | Medium     | Medium      |\n",
      "\n",
      "-| Integration Capability               | Excellent  | Good     | Good     | Fair     | Good       | Good        |\n",
      "\n",
      "-| Batch Processing                     | Yes        | No       | Yes      | No       | Yes        | No          |\n",
      "\n",
      "-| Custom Parsing Rules                 | Yes        | Limited  | Yes      | No       | Yes        | No          |\n",
      "\n",
      "-| Multilingual Support                 | Yes        | Yes      | Yes      | Yes      | Yes        | Yes         |\n",
      "\n",
      "-| OCR (Optical Character Recognition)  | Yes        | Yes      | Yes      | Yes      | Yes        | No          |\n",
      "\n",
      "-| Price (per user/month)               | $30        | $25      | $20      | $15      | $18        | $15         |\n",
      "\n",
      "-| Customer Support Rating (out of 5)   | 4.8        | 4.2      | 4.5      | 3.9      | 4.1        | 3.9         |\n",
      "\n",
      "-| Free Trial Available                 | Yes        | Yes      | No       | Yes      | No         | Yes         |\n",
      "\n",
      "-| Cloud Integration                    | Yes        | No       | Yes      | No       | No         | Yes         |\n",
      "\n",
      "-| Security Features                    | Advanced   | Basic    | Advanced | Basic    | Intermediate| Basic      |\n",
      "\n",
      "-\n",
      "\n",
      "-\n",
      "\n",
      "-| Feature                        | Tool 1              | Tool 2           | Tool 3         | Tool 4        | Tool 5           |\n",
      "\n",
      "-|--------------------------------|---------------------|------------------|----------------|---------------|------------------|\n",
      "\n",
      "-| User Community Size            | Large               | Medium           | Medium         | Small         | Medium           |\n",
      "\n",
      "-| Monthly Updates                | Yes                 | Yes              | No             | No            | No               |\n",
      "\n",
      "-| Mobile App Availability        | Yes                 | No               | Yes            | No            | No               |\n",
      "\n",
      "-| Platform Compatibility         | Windows, Mac, Linux | Windows, Linux   | Windows        | Mac, Linux    | Windows, Linux   |\n",
      "\n",
      "-| Data Privacy Compliance        | High                | Medium           | High           | Low           | Medium           |\n",
      "\n",
      "-| AI-Driven Enhancements         | Yes                 | No               | Yes            | No            | Yes              |\n",
      "\n",
      "-| File Size Limit (per document) | 1GB                 | 500MB            | 750MB          | 200MB         | 500MB            |\n",
      "\n",
      "-| User Training Resources        | Extensive           | Moderate         | Extensive      | Limited       | Moderate         |\n",
      "\n",
      "-| API Access                     | Yes                 | No               | Yes            | No            | Yes              |\n",
      "\n",
      "-| Customizable Output Templates  | Yes                 | Limited          | Yes            | No            | Limited          |\n",
      "\n",
      "-| Collaboration Features         | Yes                 | No               | Yes            | No            | Limited          |\n",
      "\n",
      "-| Document Version Control       | Yes                 | No               | Yes            | No            | Yes              |\n",
      "\n",
      "-| Import/Export Options          | Extensive           | Moderate         | Extensive      | Limited       | Moderate         |\n",
      "\n",
      "-\n",
      "\n",
      "-\n",
      "\n",
      "-| Feedback Mechanism | Yes | No | Yes | No | Yes |\n",
      "\n",
      "-|--------------------|-----|----|-----|----|-----|\n",
      "\n",
      "-\n",
      "\n",
      "+| Metric              | Mega Parse  | Parser A       | Parser B     | Parser C     | Parser D       |\n",
      "\n",
      "+|---------------------|-------------|----------------|--------------|--------------|----------------|\n",
      "\n",
      "+| Supported Formats   | PDF, DOCX, PPTX | PDF, DOCX     | DOCX, PPTX   | PDF, PPTX    | PDF, DOCX, XLSX|\n",
      "\n",
      "+| Conversion Speed (pages/min) | 120 | 90             | 100          | 85           | 95             |\n",
      "\n",
      "+| Accuracy Rate (%)     | 98   | 95   | 93   | 90   | 92   |\n",
      "\n",
      "+| Output Format         | Markdown | HTML     | Markdown | Plain Text | HTML   |\n",
      "\n",
      "+| Error Rate (%)        | 1        | 3        | 4        | 5        | 3        |\n",
      "\n",
      "+| Ease of Use           | High     | Medium   | High     | Medium   | Medium   |\n",
      "\n",
      "+| Integration Capability| Excellent| Good     | Good     | Fair     | Good     |\n",
      "\n",
      "+| Batch Processing      | Yes      | No       | Yes      | No       | Yes      |\n",
      "\n",
      "+| Custom Parsing Rules  | Yes      | Limited  | Yes      | No       | Limited  |\n",
      "\n",
      "+| Multilingual Support  | Yes      | Yes      | No       | Yes      | Yes      |\n",
      "\n",
      "+| OCR (Optical Character Recognition) | Yes | No | Yes | No | Yes |\n",
      "\n",
      "+| Price (per user/month)| $30      | $25      | $20      | $15      | $18      |\n",
      "\n",
      "+| Customer Support Rating (out of 5) | 4.8 | 4.2 | 4.5 | 3.9 | 4.1 |\n",
      "\n",
      "+| Free Trial Available  | Yes      | Yes      | No       | Yes      | No       |\n",
      "\n",
      "+| Cloud Integration     | Yes      | No       | Yes      | Yes      | No       |\n",
      "\n",
      "+| Security Features     | Advanced | Basic    | Advanced | Basic    | Intermediate |\n",
      "\n",
      "+| User Community Size   | Large    | Medium   | Medium   | Small    | Medium   |\n",
      "\n",
      "+| Monthly Updates       | Yes      | Yes      | No       | Yes      | No       |\n",
      "\n",
      "+| Mobile App Availability| Yes     | No       | Yes      | No       | Yes      |\n",
      "\n",
      "+| Platform Compatibility| Windows, Mac, Linux | Windows, Mac | Windows | Mac, Linux | Windows, Linux |\n",
      "\n",
      "+| Data Privacy Compliance| High    | Medium   | High     | Low      | Medium   |\n",
      "\n",
      "+| AI-Driven Enhancements| Yes      | No       | Yes      | No       | Yes      |\n",
      "\n",
      "+| File Size Limit (per document) | 1GB | 500MB | 750MB | 200MB | 500MB |\n",
      "\n",
      "+| User Training Resources| Extensive | Moderate | Extensive | Limited  | Moderate |\n",
      "\n",
      "+| API Access            | Yes      | No       | Yes      | No       | Yes      |\n",
      "\n",
      "+| Customizable Output Templates | Yes | Limited | Yes | No | Yes |\n",
      "\n",
      "+| Collaboration Features| Yes      | No       | Yes      | No       | Limited  |\n",
      "\n",
      "+| Document Version Control| Yes    | No       | Yes      | No       | Yes      |\n",
      "\n",
      "+| Import/Export Options | Extensive | Moderate | Extensive | Limited | Moderate |\n",
      "\n",
      "+| Feedback Mechanism    | Yes      | No       | Yes      | No       | Yes      |\n",
      "\n",
      " \n",
      "\n",
      " Note: All data presented in this table is fictional and for illustrative purposes only.\n",
      "\n",
      " \n",
      "\n",
      "-# Conclusion\n",
      "\n",
      "+## Conclusion\n",
      "\n",
      " \n",
      "\n",
      "-Mega Parse stands out as a leading document parser due to its extensive format support, high-speed processing, and accuracy. Its ability to convert a variety of document types into Markdown format makes it an invaluable tool for organizations looking to streamline their document management processes and enhance their knowledge management systems. With features like customizable parsing rules, batch processing, and advanced error handling, Mega Parse is well-equipped to meet the demands of modern enterprises. Its scalability and cost-effectiveness further reinforce its position as a top choice for document parsing and conversion needs. By leveraging Mega Parse, organizations can improve their workflow efficiency, reduce operational costs, and better manage their information assets in the age of big data and artificial intelligence.\n",
      "\n",
      "-\n",
      "\n",
      "+Mega Parse stands out as a leading document parser due to its extensive format support, high-speed processing, and accuracy. Its ability to convert a variety of document types into Markdown format makes it an invaluable tool for organizations looking to streamline their document management processes and enhance their knowledge management systems. With features like customizable parsing rules, batch processing, and advanced error handling, Mega Parse is well-equipped to meet the demands of modern enterprises. Its scalability and cost-effectiveness further reinforce its position as a top choice for document parsing and conversion needs. By leveraging Mega Parse, organizations can improve their workflow efficiency, reduce operational costs, and better manage their information assets in the age of big data and artificial intelligence.\n"
     ]
    }
   ],
   "source": [
    "source_lines = read_file(\"../megaparse/tests/output_tests/MegaFake_report_unstructured_augmented.md\")\n",
    "target_lines = read_file(\"../megaparse/tests/output_tests/MegaFake_report.md\")\n",
    "\n",
    "source_lines = [line.replace(\"*\",\"\") for line in source_lines]\n",
    "target_lines = [line.replace(\"*\",\"\") for line in target_lines]\n",
    "\n",
    "diff = difflib.unified_diff(\n",
    "source_lines,\n",
    "target_lines,\n",
    "fromfile='target.md',\n",
    "tofile='generated.md',\n",
    "lineterm=''\n",
    ")\n",
    "modifications = 0\n",
    "for line in diff:\n",
    "    print(line)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "QuivrParse-DS8JDGq8",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
